﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Configuration;
using System.IO;
using System.Xml;
using System.Xml.Linq;
using Lucandra.Net;
using Lucene.Net.Documents;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using System.Diagnostics;
using Lucene.Net.Search;
using Lucene.Net.QueryParsers;

namespace Wikipedia_Example
{
    class Program
    {
        static long position = 0;
        static FileStream positionWriter = null;
        static LucandraWriter lucandraWriter = null;
        static bool storeArticleText = false;
        static Analyzer analyzer;
        static string indexName;
        static string filePath;

        static void Main(string[] args)
        {
            if (!LoadConfig())
                goto FINISH;

            // Create the analyzer to be used for tokenizing terms.
            analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29);

            string[] options = new string[] { "1", "2", "9" };

            Console.WriteLine("Would you like to index or search?");
            Console.WriteLine();
            Console.WriteLine("  [1] Index");
            Console.WriteLine("  [2] Search");
            Console.WriteLine("  [9] Exit");

            string option = null;
            while (!options.Contains(option))
            {
                if (option != null)
                    Console.WriteLine("Invalid option!");

                Console.WriteLine();
                Console.Write("> ");
                option = Console.ReadLine();
            }

            switch (option)
            {
                case "1": 
                    DoIndexing(); 
                    break;
                case "2": 
                    DoSearching(); 
                    break;
                case "9":
                    goto FINISH;
            }

        FINISH:
            Console.WriteLine();
            Console.WriteLine("Press any key to exit...");
            Console.ReadLine();
        }

        static void DoSearching()
        {
            var reader = new LucandraReader(indexName);
            var searcher = new IndexSearcher(reader);
            var parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, "title", analyzer);

            string[] enumerableFields = new string[] { "page_id", "page_title" };

            string queryText = null;

            while (true)
            {
                try
                {
                    Console.WriteLine();
                    Console.Write("Query> ");
                    queryText = Console.ReadLine();
                    if (queryText == "exit")
                        break;

                    var query = parser.Parse(queryText);
                    var sw = Stopwatch.StartNew();
                    var docs = searcher.Search(query, 10);
                    sw.Stop();

                    Console.WriteLine("Search completed in {0}.  {1} documents returned/{2} total hits.", sw.Elapsed, docs.scoreDocs.Length, docs.totalHits);
                    if (docs.scoreDocs.Length > 0)
                    {
                        Console.Write("Enumerate results? (Y/N): ");
                        string enumerate = Console.ReadLine();
                        if (enumerate.ToLower() == "y")
                        {
                            for (int i = 0; i < docs.scoreDocs.Length; i++)
                            {
                                var doc = searcher.Doc(docs.scoreDocs[i].doc);
                                Console.WriteLine();
                                Console.WriteLine("Document {0} (#{1}) - Score: {2}", i, docs.scoreDocs[i].doc, docs.scoreDocs[i].score);
                                foreach (Field field in doc.GetFields())
                                {
                                    if (enumerableFields.Contains(field.Name()))
                                        Console.WriteLine("   * {0} = {1}", field.Name(), field.StringValue());
                                }
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Error: " + ex.Message);
                }
            }
        }

        static void DoIndexing()
        {
            position = 0;
            // Check if a position file exists (used to keep track of the <page> element that we last indexed).
            if (File.Exists("position"))
            {
                // Read the position of the last-indexed document.
                var positionBytes = File.ReadAllBytes("position");
                if (positionBytes.Length == 8)
                {
                    position = BitConverter.ToInt64(positionBytes, 0);

                    // Increment the position so that we start at the next page.
                    position++;
                }
            }

            // Open the writer for the position file.
            positionWriter = File.Open("position", FileMode.OpenOrCreate, FileAccess.Write, FileShare.None);

            // Create the LucandraWriter.
            lucandraWriter = new LucandraWriter(indexName);

            // Open the wikipedia index XML document and begin iterating through and indexing documents.
            using (var fileStream = new FileStream(filePath, FileMode.Open))
            {
                using (var streamReader = new StreamReader(fileStream))
                {
                    using (var xmlReader = new XmlTextReader(streamReader))
                    {
                        long count = 0;

                        while (xmlReader.ReadToFollowing("page"))
                        {
                            if (count < position)
                            {
                                count++;
                                continue;
                            }

                            if (xmlReader.NodeType == XmlNodeType.Element && !xmlReader.IsEmptyElement)
                            {
                                var outerXml = xmlReader.ReadOuterXml();
                                var doc = XDocument.Parse(outerXml);
                                var root = doc.Root;
                                string ns = "http://www.mediawiki.org/xml/export-0.4/";

                                // Parse the XML for the page into a usable object.
                                var page = new WikiPage()
                                {
                                    ID = root.Element(XName.Get("id", ns)).Value,
                                    Title = root.Element(XName.Get("title", ns)).Value,
                                    Revisions = new List<WikiPageRevision>()
                                };

                                foreach (var revision in root.Elements(XName.Get("revision", ns)))
                                {
                                    var contributor = revision.Element(XName.Get("contributor", ns));
                                    page.Revisions.Add(new WikiPageRevision()
                                    {
                                        ID = revision.Element(XName.Get("id", ns)).Value,
                                        Comment = revision.Element(XName.Get("comment", ns)) == null ? null : revision.Element(XName.Get("comment", ns)).Value,
                                        Timestamp = DateTime.Parse(revision.Element(XName.Get("timestamp", ns)).Value),
                                        Contributor = new WikiContributor()
                                        {
                                            ID = contributor.Element(XName.Get("id", ns)) == null ? null : contributor.Element(XName.Get("id", ns)).Value,
                                            Username = contributor.Element(XName.Get("username", ns)) == null ? null : contributor.Element(XName.Get("username", ns)).Value,
                                            IPAddress = contributor.Element(XName.Get("ip", ns)) == null ? null : contributor.Element(XName.Get("ip", ns)).Value
                                        },
                                        Text = revision.Element(XName.Get("text", ns)).Value
                                    });
                                }

                                // Index the wiki page.
                                IndexArticle(page);

                                count++;
                            }
                        }
                    }
                }
            }
        }

        static void IndexArticle(WikiPage page)
        {
            foreach (var revision in page.Revisions)
            {
                // Create the document and add applicable fields from the wiki page.
                var doc = new Document();

                doc.Add(new Field("page_id", page.ID, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                doc.Add(new Field("page_title", page.Title, Field.Store.YES, Field.Index.ANALYZED));
                doc.Add(new Field("rev_id", revision.ID, Field.Store.YES, Field.Index.NO));
                doc.Add(new Field("rev_timestamp", revision.Timestamp.Ticks.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                if (!string.IsNullOrWhiteSpace(revision.Comment))
                    doc.Add(new Field("rev_comment", revision.Comment, Field.Store.YES, Field.Index.NO));
                doc.Add(new Field("rev_text", revision.Text, storeArticleText ? Field.Store.YES : Field.Store.NO, Field.Index.ANALYZED));
                if (revision.Contributor.ID != null)
                    doc.Add(new Field("rev_contrib_id", revision.Contributor.ID, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                if (revision.Contributor.Username != null)
                    doc.Add(new Field("rev_contrib_username", revision.Contributor.Username, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                if (revision.Contributor.IPAddress != null)
                    doc.Add(new Field("rev_contrib_ip", revision.Contributor.IPAddress, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));

                // Add the document revision to the index.
                var sw = Stopwatch.StartNew();
                lucandraWriter.AddDocument(doc, analyzer);
                sw.Stop();

                if (position % 100 == 0)
                    Console.WriteLine("Position {0} indexed in {1}.", position, sw.Elapsed);
            }

            // Increment the current position in the position file.
            positionWriter.Position = 0;
            var positionBytes = BitConverter.GetBytes(position);
            positionWriter.Write(positionBytes, 0, positionBytes.Length);

            // Increment the position to the next document.
            position++;
        }

        static bool LoadConfig()
        {
            filePath = ConfigurationManager.AppSettings["WikipediaXmlDumpFilePath"];
            if (string.IsNullOrWhiteSpace(filePath) || !File.Exists(filePath))
            {
                Console.WriteLine("The 'WikipediaXmlDumpFilePath' application setting is either not configured or the file does not exist.");
                return false;
            }

            indexName = ConfigurationManager.AppSettings["IndexName"];
            if (string.IsNullOrWhiteSpace(indexName))
            {
                Console.WriteLine("The 'IndexName' application setting is not configured.");
                return false;
            }

            string storeArticleTextString = ConfigurationManager.AppSettings["StoreArticleText"];
            if (!string.IsNullOrWhiteSpace(storeArticleTextString) || storeArticleTextString.ToLower() == "true")
                storeArticleText = true;

            return true;
        }
    }
}
